#!/usr/bin/env python3
import json
import math
import sqlite3
import time
from functools import reduce
import requests
from lxml import etree


class NewsIndex:
    """## 新闻索引类

    包含所有从api中返回的信息
    """

    def __init__(self, construct_dict: dict):
        self.__dict__.update(construct_dict)
        self.timestamp = int(construct_dict['createtime'])
        self.title = construct_dict["title"]
        self.id = construct_dict['id']
        self.media = construct_dict['media_name']
        self.column = construct_dict['column']
        self.url = construct_dict['url']
        self.img = construct_dict['img']
        self.ftime = time.strftime(
            "%y/%m/%d %H:%M", time.localtime(self.timestamp))

    def getftime(self):
        """获取并更新自身的格式化时间,依赖于自身的timestamp

        Returns:
            [str] -- [description]
        """

        self.ftime = time.strftime(
            "%y/%m/%d %H:%M", time.localtime(self.timestamp))
        return self.ftime

    def __str__(self):
        return "{{{} {}}}".format(self.title, self.ftime)

    def __repr__(self):
        return "{{{},{}}}".format(self.id, self.ftime)


class NewsContent:
    """## NewsContent
    新闻内容主体
    """

    def __init__(self, index: NewsIndex):
        """

        :param index:
        """
        self.index = index
        self.title = index.title
        self.content = None
        self.success = True

    def fetch_content(self):
        """## fetch_content()
        通过自身的NewsIndex类中的信息`url`获取新闻正文内容

        """
        try:

            res = requests.get(self.index.url)
            res.encoding = 'utf-8'
        except requests.exceptions.RequestException as requestfail:
            print("requests异常!  ", requestfail)
            self.content = ''
            self.success = False
            return
        tree = etree.HTML(res.text)
        paragraph = tree.xpath('//*[@class="article" or @id="article" or @id="artibody"]//p//child::text()')
        try:
            self.content = reduce(lambda x, y: x + '\n' + y, paragraph)
        except TypeError as typeerror:
            print("TypeError here!  ", typeerror)
            print(paragraph)
            self.content = ''
            self.success = False
            return

    def totuple(self):
        """## to tuple

        返回一个元组类型,包含信息的顺序为
        `(id, url, timestamp, media, title, content)`
        """
        return (self.index.id,
                self.index.url,
                self.index.timestamp,
                self.index.media,
                self.index.title,
                self.content,
                )

    @classmethod
    def fromtuple(cls, t: tuple):
        """## from tuple

        用元组构建这个对象

        Arguments:
            t {tuple} -- 构建对象的元组
        """
        newobj = cls(NewsIndex(dict(createtime=0,
                                    title=None,
                                    id=None,
                                    media_name=None,
                                    column=None,
                                    url=None,
                                    img=None,
                                    ftime=None)
                               ))
        try:
            newobj.index.id, newobj.index.url, newobj.index.timestamp, newobj.index.media, newobj.index.title, newobj.content = t

        except TypeError as typeerror:
            print(typeerror, t)
        except Exception as e:
            print("unkonw ", e, t)
        newobj.title = newobj.index.title
        return newobj

    def __str__(self):
        if self.content is None:
            content = None
        elif len(self.content) > 30:
            content = self.content[:30]
        else:
            content = self.content
        return "{{{} {} \n {} }}".format(self.title, self.index.ftime, content)

    def __repr__(self):
        return "{{NewsContent id:{} time:{} }}".format(self.index.id, self.index.timestamp)


def getlist(start_time, quantity=10, page=1, cat='=gdxw1||=gatxw||=zs-pl||=mtjj', callback_str=None):
    """## 获取新闻列表
    # Arguments:
        `start_time` {int} -- unix时间戳, 精确到秒

    # Keyword Arguments:
        `quantity` {int} -- 每页的数量 (default: {100})

        `page` {int} -- 页序号,即以quantity(show_num)为数量的页序号(default: {1})
        例如,quantity(show_num)为10,page 为2 则请求到的数据为从当前时间往前数第20-30条新闻
            quantity(show_num)为10,page 为1 则请求到的数据为从当前时间往前数第1-10条新闻
            quantity(show_num)叫做每页的数量比较合适

        `cat` {str} -- 类型 (default: {'=gdxw1||=gatxw||=zs-pl||=mtjj'})

        `callback_str` {str} -- 回调字符串,非空会在外面包裹一层函数名 (default: {None})

    # Returns:
        {list} 字典数组,失败返回None
    """

    url = "http://api.roll.news.sina.com.cn/zt_list"
    # unused = "=newsloadercallback&_=1513428517160"
    # "gdxw1||=gatxw||=zs-pl||=mtjj"
    params = dict(
        channel="news",
        cat_1="gnxw",
        cat_2=cat,
        level="=1||=2",
        show_ext="1",
        show_all="1",
        show_num=quantity,
        tag="1",
        format="json",
        page=page,
        _=int(start_time),
        # callback = callback_str
    )
    res = requests.get(url, params=params)
    # print(res.url)
    try:
        res_json = json.loads(res.text)
        result = res_json["result"]
        resmsg = '''返回信息为:
        'status' : {},
        'encoding':{},
        'serverSeconds': {},
        'total': {},
        'count': {},
        'last_time': {}'''
        print("获取到{}条原始引索数据".format(len(result['data'])))
        print(resmsg.format(
            result['status'],
            result['encoding'],
            result['serverSeconds'],
            result['total'],
            result['count'],
            result['last_time']))

        return result["data"]
    except json.JSONDecodeError as decodefialed:
        print(decodefialed)
        print(res)
        return ''
    except IndexError:
        print("索引异常")
        print(res_json)
        return ''


class NewsDatabase:
    """新闻数据库
    """

    def __init__(self,  dbname='data.db'):
        self.dbname = dbname
        self.conn = sqlite3.connect(self.dbname, isolation_level=None)
        self.conn.execute('''CREATE TABLE IF NOT EXISTS news(
            id INT PRIMARY KEY     NOT NULL,
            url TEXT ,
            timestamp INT ,
            media TEXT ,
            title TEXT ,
            content TEXT
        )''')
        self.conn.execute('''CREATE TABLE IF NOT EXISTS fail(
            id INT PRIMARY KEY     NOT NULL,
            url TEXT ,
            timestamp INT
        )''')
        self.conn.execute('''CREATE TABLE IF NOT EXISTS fail_twice(
            id INT PRIMARY KEY     NOT NULL,
            url TEXT ,
            timestamp INT
        )''')

    def savetodb(self, items):
        """## savetodb(items)
        将items中的数据保存到数据库
        # Arguments:
            `items` {list} -- 元组的列表
        """
        try:
            if isinstance(items, tuple):
                self.conn.execute(
                    'INSERT INTO news VALUES (?,?,?,?,?,?)', items)
            else:
                self.conn.executemany(
                    'INSERT INTO news VALUES (?,?,?,?,?,?)', items)
        except sqlite3.Error as sqlfail:
            print(sqlfail)
            return False
        return True

    def savetofail(self, news: NewsContent):
        """>将获取失败的数据存进数据库里
        # Arguments:
            `news` {NewsContent} -- NewsContent对象

        Returns:
            bool  -- 返回是否保存成功
        """
        try:
            self.conn.execute('INSERT INTO fail VALUES (?,?,?)',
                              (news.index.id, news.index.url, news.index.timestamp))
        except sqlite3.Error as sqlfail:
            print(sqlfail)
            return False
        return True

    def getlatest(self):
        """返回当前数据库中最新的有效数据
        Returns:
            NewsContent对象
        """
        latest_cur = self.conn.execute(
            'SELECT * FROM news ORDER BY news.timestamp DESC LIMIT 1')
        try:

            latest = latest_cur.fetchone()
        except TypeError as tperr:
            latest = None
            print("Type Error, maybe empty database:", tperr)
            return None
        return NewsContent.fromtuple(latest)

    def getoldest(self):
        """返回当前数据库中最旧的有效数据
        Returns:
            NewsContent对象
        """
        old_cur = self.conn.execute(
            'SELECT * FROM news ORDER BY news.timestamp ASC LIMIT 1')
        try:

            old = old_cur.fetchone()
        except TypeError as tperr:
            old = None
            print("Type Error, maybe empty database:", tperr)
            return None
        return NewsContent.fromtuple(old)

    def newscount(self):
        """返回当前数据库中有效数据的数量
        """
        cnt = 0
        try:
            cnt = self.conn.execute("SELECT count(*) FROM news").fetchone()[0]
        except sqlite3.Error as sqlfial:
            print("Sqlite fail:", sqlite3)
        except TypeError as typeerror:
            print("Type Error, maybe empty database:", typeerror)
        return cnt
    def failcount(self):
        """返回当前数据库中获取失败的数量
        """
        cnt = 0
        try:
            cnt = self.conn.execute("SELECT count(*) FROM fail").fetchone()[0]
        except sqlite3.Error as sqlfial:
            print("Sqlite fail:", sqlite3)
        except TypeError as typeerror:
            print("Type Error, maybe empty database:", typeerror)
        return cnt

    def getfails(self, limit=None):
        """获取数据库中失败的索引
        Returns:
            NewsIndex对象的列表
        """

        limit_str = "LIMIT {}".format(int(limit))
        ret = []
        try:
            failed_indexlist = self.conn.execute(
                "SELECT * FROM fail {} ".format(limit_str if limit != None else " ")).fetchall()
            for fail in failed_indexlist:
                ret.append(NewsIndex(dict(createtime=fail[2],
                                          title=None,
                                          id=fail[0],
                                          media_name=None,
                                          column=None,
                                          url=fail[1],
                                          img=None,
                                          ftime=None)
                                     ))
        except sqlite3.Error as sqlfial:
            print("Sqlite fail:", sqlfial)
        except TypeError as typeerror:
            print("Type Error, maybe empty database:", typeerror)

        return ret

    def getfailone(self):
        """获取一条数据库中失败的索引
        Returns:
            NewsIndex对象
        """
        try:
            return self.getfails(1)[0]
        except IndexError:
            return None

    def dropfail(self, fail_id):
        """将id为fail_id的条目从fail表中删除
        Arguments:
            fail_id {str} -- 要删除的条目的id
        """

        try:
            self.conn.execute("DELETE FROM fail WHERE id = (?)", (fail_id,))
        except sqlite3.Error as sqlfial:
            print("Sqlite fail:", sqlfial)
            return False
        return True
    def retryfail(self,  news: NewsContent):
        """>将 重试之后仍然 获取失败的数据存进数据库里
        # Arguments:
            `news` {NewsContent} -- NewsContent对象

        Returns:
            bool  -- 返回是否保存成功
        """
        try:
            self.conn.execute('INSERT INTO fail_twice VALUES (?,?,?)',
                              (news.index.id, news.index.url, news.index.timestamp))
        except sqlite3.Error as sqlfail:
            print(sqlfail)
            return False
        return True
    def close(self):
        """关闭连接
        """

        self.conn.close()


def main(fetch_eachtime=1000):
    db = NewsDatabase("news.db")
    #获取 数据库中最新和最旧的条目
    latest = db.getlatest()
    oldest = db.getoldest()
    #显式更新时间字符串
    latest.index.getftime()
    oldest.index.getftime()
    show_databasemsg = '''
当前数据库存有:{n}条成功数据, {fail}条失败数据
其中最新的为:
{latest}
最旧的为:
{oldest}
    '''
    print(show_databasemsg.format(n=db.newscount(),fail =db.failcount() , latest=latest, oldest=oldest))
    #获取需要数量
    num = int(input("请输入需要的新闻数量:"))
    #  total_times 总共要获取的次数, num/fetch_eachtime向上取整,
    # 因为用了取整,所以每次获取的总数理论上都会大于等于需要获取的量
    total_times = math.ceil(num / fetch_eachtime)
    page_fetched = 0
    start_time = time.time()
    while page_fetched < total_times:
        page_fetched += 1
        print("准备获取从{}以前的{}条数据".format(time.strftime(
            "%y/%m/%d %H:%M", time.localtime(start_time)), fetch_eachtime))
        s = getlist(start_time, fetch_eachtime, page=page_fetched)

        # print(s)

        latest = db.getlatest()
        oldest = db.getoldest()
        for i in s:
            news = NewsContent(NewsIndex(i))
            # 时间无需更改
            # start_time = news.index.timestamp
            # if latest and oldest:
            #     if news.index.timestamp < latest.index.timestamp and news.index.timestamp > oldest.index.timestamp:
            #         #"比数据库里最新的旧 or 比数据库里最旧的新 "
            #         continue

            # 延时0.5s
            time.sleep(0.5)
            news.fetch_content()

            print(news)
            if news.success:
                db.savetodb(news.totuple())
            else:
                db.savetofail(news)

        latest.index.getftime()
        oldest.index.getftime()
        print(show_databasemsg.format(n=db.newscount(),fail =db.failcount() , latest=latest, oldest=oldest))
    #   对失败数据进行重试
    while db.failcount()!=0:
        failindex = db.getfailone()
        news = NewsContent(failindex)
        print(" 正在对id:{}进行重试...".format(news.index.id))
        time.sleep(0.5)
        news.fetch_content()

        print(news)
        if news.success:
            db.savetodb(news.totuple())
        else:
            print("id:{}依旧获取失败".format(news.index.id))
            db.retryfail(news)
        db.dropfail(news.index.id)
    db.close()
    # return news


if __name__ == "__main__":
    main()
